capstone
Chris Snow
06/11/2016
Load the raw data
library(tools)
md5 <- md5sum('Coursera-SwiftKey.zip')
if (md5 != 'e0629c64b1747103a7e751b0a30d3858') {
url <- 'https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip'
download.file(url, destfile = 'Coursera-SwiftKey.zip')
unzip('Coursera-SwiftKey.zip')
# free some space on my drive by removing unneeded files
unlink('./final/de_DE/', recursive = TRUE, force = TRUE)
unlink('./final/fi_FI/', recursive = TRUE, force = TRUE)
unlink('./final/ru_RU/', recursive = TRUE, force = TRUE)
}Load required packages
if(!require(slam)) {
install.packages("slam", type="binary")
library(slam)
}
if(!require(tm)) {
install.packages("tm")
library(tm)
}
if(!require(Rgraphviz)) {
source("http://bioconductor.org/biocLite.R")
biocLite("Rgraphviz", ask = FALSE)
library(Rgraphviz)
}
if(!require(wordcloud)) {
install.packages("wordcloud")
library(wordcloud)
}Create the corpus
bannedWords <- as.list(read.csv(url('http://www.bannedwordlist.com/lists/swearWords.txt'), header = FALSE))
# head(bannedWords)LIMIT_NUM_DOCS <- 5000
con <- file('./final/en_US/en_US.blogs.txt')
open(con)
v <- readLines(con, n=LIMIT_NUM_DOCS)
close(con)
vs <- VectorSource(v)
removeWordsContaining <-
function(x, words)
UseMethod("removeWordsContaining", x)
removeWordsContaining.character <-
function(x, words)
gsub(sprintf("(*UCP)\\b[a-zA-Z]*(%s)[a-zA-Z]*\\b",
paste(sort(words, decreasing = TRUE), collapse = "|")),
"", x, perl = TRUE)
removeWordsContaining.PlainTextDocument <-
content_transformer(removeWordsContaining.character)
blog_corpus <- Corpus(vs, readerControl = list(language="en"))
blog_corpus <- tm_map(blog_corpus, content_transformer(tolower))
blog_corpus <- tm_map(blog_corpus, stripWhitespace)
blog_corpus <- tm_map(blog_corpus, removePunctuation)
blog_corpus <- tm_map(blog_corpus, removeNumbers)
#blog_corpus <- tm_map(blog_corpus, removeWords, c(stopwords("english")))
blog_corpus <- tm_map(blog_corpus, removeWordsContaining, bannedWords$V1) Tokenize to plain TermDocumentMatrix
if (LIMIT_NUM_DOCS < 200)
wordcloud(blog_corpus)tdm <- TermDocumentMatrix(blog_corpus,
control = list(removePunctuation = TRUE,
removeNumbers = TRUE,
stopwords = TRUE))
inspect(tdm[1:10, 1:10])## <<TermDocumentMatrix (terms: 10, documents: 10)>>
## Non-/sparse entries: 0/100
## Sparsity : 100%
## Maximal term length: 7
## Weighting : term frequency (tf)
##
## Docs
## Terms 1 2 3 4 5 6 7 8 9 10
## ½in 0 0 0 0 0 0 0 0 0 0
## aadil 0 0 0 0 0 0 0 0 0 0
## aaggh 0 0 0 0 0 0 0 0 0 0
## aamoth 0 0 0 0 0 0 0 0 0 0
## aand 0 0 0 0 0 0 0 0 0 0
## aaronic 0 0 0 0 0 0 0 0 0 0
## aarons 0 0 0 0 0 0 0 0 0 0
## aasif 0 0 0 0 0 0 0 0 0 0
## aasl 0 0 0 0 0 0 0 0 0 0
## aayeaye 0 0 0 0 0 0 0 0 0 0terms <- dimnames(tdm[ ,1:50])$Termsinspect(tdm[grep('aadil', terms), 1:10])## <<TermDocumentMatrix (terms: 1, documents: 10)>>
## Non-/sparse entries: 0/10
## Sparsity : 100%
## Maximal term length: 5
## Weighting : term frequency (tf)
##
## Docs
## Terms 1 2 3 4 5 6 7 8 9 10
## aadil 0 0 0 0 0 0 0 0 0 0inspect(tdm[grep('fuck', terms), 1:10])## <<TermDocumentMatrix (terms: 0, documents: 10)>>
## Non-/sparse entries: 0/0
## Sparsity : 100%
## Maximal term length: 0
## Weighting : term frequency (tf)
##
## Docs
## Terms 1 2 3 4 5 6 7 8 9 10plot(tdm, corThreshold = 0.2, weighting = TRUE)
Tokenize to ngrams - What are the frequencies of 2-grams and 3-grams in the dataset?
ngramDTM <- function (ngrams) {
TrigramTokenizer <-
function(x)
unlist(lapply(ngrams(words(x), ngrams), paste, collapse = " "), use.names = FALSE)
datmat <- TermDocumentMatrix(blog_corpus, control = list(tokenize = TrigramTokenizer))
return(as.matrix(datmat))
}digrams = ngramDTM(2)
digrams[1:5, 1:3]## Docs
## Terms 1 2 3
## ½ seconds 0 0 0
## ½in cookie 0 0 0
## ¼ teaspoon 0 0 0
## ⅛ teaspoon 0 0 0
## a adapt 0 0 0digramsCounts <- rowSums(digrams)
head(as.matrix(digramsCounts))## [,1]
## ½ seconds 1
## ½in cookie 1
## ¼ teaspoon 1
## ⅛ teaspoon 1
## a adapt 1
## a allgrain 1tail(as.matrix(digramsCounts))## [,1]
## zulfar hussain 1
## zulueta have 1
## zuma ended 1
## zwick who 1
## zydeco is 1
## それじゃ pardon 1digramsGT10 <- digramsCounts[ digramsCounts > 10 ]
as.matrix (
digramsGT10[ head( order( digramsGT10, decreasing = T), 25 ) ]
)## [,1]
## of the 1001
## in the 847
## to the 504
## to be 387
## on the 386
## for the 335
## and the 330
## and i 286
## it was 266
## with the 266
## at the 263
## it is 253
## i was 252
## i have 249
## i am 244
## in a 237
## is a 234
## from the 231
## that i 197
## of a 192
## but i 191
## with a 186
## this is 176
## for a 174
## one of 172trigrams = ngramDTM(3)
trigramsCounts <- rowSums(trigrams)
trigramsGT10 <- trigramsCounts[ trigramsCounts > 10 ]
as.matrix (
trigramsGT10[ head( order( trigramsGT10, decreasing = T), 25 ) ]
)## [,1]
## a lot of 83
## one of the 76
## some of the 39
## to be a 39
## i want to 36
## it was a 36
## i have to 33
## the end of 32
## a couple of 31
## out of the 31
## i have a 30
## as well as 29
## i had to 29
## this is a 29
## i had a 28
## the rest of 28
## i dont know 27
## part of the 27
## going to be 25
## i decided to 25
## i have been 25
## i wanted to 25
## it is a 25
## one of my 25
## i am not 24How many unique words do you need in a frequency sorted dictionary to cover 50% of all word instances in the language? 90%?
This is a sentence to understand how to count word instances and perform a frequency analysis.
- a
- to
- This
- is
- sentence
- understand
- how
- count
- word
- instances
- and
- perform
- frequency
- analysis
## <<TermDocumentMatrix (terms: 20595, documents: 5000)>>